import pandas as pd
import numpy as np

data_train = pd.read_csv('data_train.txt', index_col=[0], header=0)
data_predict = pd.read_csv('data_predict.txt', index_col=[0], header=0)


def is_noise(data_train, data_test):
    # data_train 需要标记出noise的数据，以便建模
    data_train['is_noise'] = 0
    data_train['is_noise'][data_train['non_ch'] == 1] = 1
    data_train['is_noise'][data_train['length_all'] <= 5] = 1
    data_train['is_noise'][data_train['auto'] == 1] = 1
    data_train['is_noise'][data_train['interaction'] == 1] = 1
    data_train['is_noise'][data_train['vote'] == 1] = 1
    data_train['is_noise'][data_train['ad'] == 1] = 1

    # data_test 需要标记出noise的数据，以便预测
    data_test['is_noise'] = 0
    data_test['is_noise'][data_test['non_ch'] == 1] = 1
    data_test['is_noise'][data_test['length_all'] <= 5] = 1
    data_test['is_noise'][data_test['auto'] == 1] = 1
    data_test['is_noise'][data_test['interaction'] == 1] = 1
    data_test['is_noise'][data_test['vote'] == 1] = 1
    data_test['is_noise'][data_test['ad'] == 1] = 1

    # data_test的用户数据需要从data_train中获取
    user_history = data_train[
        ['user_id', 'number_in_train', 'forward_max', 'comment_max', 'like_max', 'forward_min', 'comment_min',
         'like_min', 'forward_mean', 'comment_mean', 'like_mean', 'forward_more_ave_pr', 'comment_more_ave_pr',
         'like_more_ave_pr', 'max_f/l', 'max_c/l', 'min_f/l', 'min_c/l', 'mean_f/l', 'mean_c/l']]
    user_history = user_history.drop_duplicates()
    data_test = pd.merge(data_test, user_history, on='user_id', how='left', indicator=False)
    data_test = data_test.fillna(-1)  # 将缺失用户的数值标记为-1，为了与互动数为0的用户区分开来
    # 如果data_test中出现新用户，标记为noise值
    data_test['is_noise'][data_test['number_in_train'] == -1] = 1
    # 如果data_test中的旧用户的互动数为0，标记为noise值
    data_test['is_noise'][(data_test['forward_max'] + data_test['comment_max'] + data_test['like_max']) == 0] = 1
    return data_train, data_test


data_train, data_valid = is_noise(data_train, data_predict)

data_train.to_csv('data_train.txt', header=True)
data_predict.to_csv('data_predict.txt', header=True)

data2 = pd.read_csv('data_train.txt', index_col=[0], header=0)
data2 = pd.read_csv('data_predict.txt', index_col=[0], header=0)
data2.head()

data2.columns
